In [2]:
%%HTML
<style type="text/css">
table.dataframe td, table.datafreame th {
border: 1px black solid;
color: black;
}
In [6]:
df.shape
Out[6]:
In [7]:
df.info()
In [8]:
df.isnull().sum()
Out[8]:
In [9]:
df.drop(columns = ['tweet_id', 'name'], axis = 1, inplace = True)
In [10]:
df.columns
Out[10]:
finding the sentiment for each tweets¶
In [11]:
from nltk.sentiment.vader import SentimentIntensityAnalyzer
In [12]:
sid = SentimentIntensityAnalyzer()
In [13]:
sid.polarity_scores(df.text[0])
Out[13]:
In [14]:
df['polarity_score'] = df.text.apply(lambda x: sid.polarity_scores(x))
In [15]:
df.head()
Out[15]:
In [16]:
df['score'] = df.polarity_score.apply(lambda x: x['compound'])
In [17]:
def find_sentiment(x):
if x > 0:
return "Positive"
elif x < 0:
return "Negative"
else:
return "Neutral"
In [18]:
df["sentiment"] = df.score.apply(lambda x: find_sentiment(x))
In [19]:
df.head()
Out[19]:
now we can drop polaarity score and score column¶
In [20]:
df.drop(columns = ['polarity_score', 'score'], axis = 1, inplace = True)
In [21]:
df.head()
Out[21]:
In [22]:
df.sentiment.value_counts()
Out[22]:
In [23]:
sns.countplot(x = 'sentiment', data = df)
Out[23]:
In [24]:
fig = go.Figure()
c = ['green','red','blue']
fig.add_trace(go.Bar(x = df.sentiment.value_counts().index,
y = df.sentiment.value_counts(),
marker_color = c,
text = df.sentiment.value_counts(),
textposition='auto'))
fig.update_layout(title = dict(text = "Sentiment count"))
Topic Modeling of the tweets¶
In [25]:
import spacy
In [26]:
nlp = spacy.load('en_core_web_sm')
In [27]:
def tokenization(s):
text = nlp(s.lower())
token = []
for t in text:
token.append(t.lemma_)
return token
In [28]:
df["lemma"] = df.text.apply(lambda x: tokenization(x))
In [29]:
df.head()
Out[29]:
In [30]:
def stopwords(s):
l = []
for i in s:
if i not in nlp.Defaults.stop_words:
l.append(i)
return " ".join(l)
In [31]:
df['proccesed_text'] = df.lemma.apply(lambda x: stopwords(x))
In [32]:
from sklearn.feature_extraction.text import TfidfVectorizer
In [33]:
tdf = TfidfVectorizer(max_df=0.9, min_df=10)
In [34]:
dtm = tdf.fit_transform(df["proccesed_text"])
In [35]:
from sklearn.decomposition import LatentDirichletAllocation
In [36]:
lda = LatentDirichletAllocation(n_components= 10, random_state= 0)
In [37]:
lda.fit(dtm)
Out[37]:
In [38]:
for i, topic in enumerate(lda.components_):
print(f"Top 20 words for topic {i}")
print([tdf.get_feature_names()[index] for index in topic.argsort()[-20:]])
print("\n")
In [39]:
topic = lda.transform(dtm)
In [40]:
df["topic"] = topic.argmax(axis = 1)
In [41]:
df.head()
Out[41]:
In [42]:
df.drop(columns=['lemma', 'proccesed_text'], axis =1, inplace = True)
In [43]:
df.head()
Out[43]:
In [44]:
reason = df.negativereason.value_counts()
In [45]:
fig = go.Figure()
fig.add_trace(go.Bar(x = reason.index,
y = reason))
In [46]:
df.retweet_count.unique()
Out[46]:
Word cloud¶
In [47]:
df_neg = df[df['sentiment'] == 'Negative']
df_neg.head()
Out[47]:
In [48]:
df_neg.to_csv (r'negativeTweets.csv', index = False, header=True)
In [49]:
import collections
import numpy as np
import pandas as pd
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from matplotlib import rcParams
from wordcloud import WordCloud, STOPWORDS
%matplotlib inline
from nltk.tokenize.treebank import TreebankWordDetokenizer
from nltk import word_tokenize
In [50]:
def removeAirline (userTyped):
tokens = word_tokenize(userTyped)
res = ''
if tokens[0] == '@':
tokens.pop(0)
tokens.pop(0)
res = TreebankWordDetokenizer().detokenize(tokens)
else:
res = userTyped
return res
In [51]:
def wordCloud_res(reason):
df_res = df_neg[df_neg['negativereason'] == reason]
all_headlines = ' '
for line in df_res['text'].str.lower():
all_headlines += removeAirline(line)
stopwords = STOPWORDS
stopwords.add('will')
wordcloud = WordCloud(stopwords=stopwords, background_color="white", max_words=500).generate(all_headlines)
print('With the reason of: ' + reason)
rcParams['figure.figsize'] = 10, 20
plt.imshow(wordcloud)
plt.axis("off")
plt.show()
In [52]:
reason = df_neg['negativereason'].unique()
reason
Out[52]:
In [53]:
wordCloud_res(reason[0])
In [54]:
wordCloud_res(reason[3])
In [55]:
wordCloud_res(reason[4])
In [56]:
wordCloud_res(reason[5])
In [57]:
wordCloud_res(reason[6])
In [58]:
wordCloud_res(reason[7])
In [59]:
wordCloud_res(reason[8])
In [60]:
wordCloud_res(reason[9])
In [61]:
wordCloud_res(reason[10])
In [ ]: